import feapder
from feapder import Item



class DangDangBook(object):

    def __init__(self, num: int, details_url: str, img_url: str, tuijian: str, publisher_author_nationality: str,
                 publisher_author_name: str, publisher: str, prce: str, title_descript: str, descript: str):
        self.num = num
        self.details_url = details_url
        self.img_url = img_url
        self.tuijian = tuijian
        self.publisher_author_nationality = publisher_author_nationality
        self.publisher_author_name = publisher_author_name
        self.publisher = publisher
        self.prce = prce
        self.title_descript = title_descript
        self.descript = descript

    def set_title_descript(self, val):
        self.title_descript = val

    def set_descript(self, val):
        self.descript = val


class Spider(feapder.AirSpider):
    __custom_setting__ = dict(
        ITEM_PIPELINES=["feapder.pipelines.mongo_pipeline.MongoPipeline"],
        MONGO_IP="server.windows",
        MONGO_PORT=27017,
        MONGO_DB="feapder",
        MONGO_USER_NAME="root",
        MONGO_USER_PASS="141535",
    )

    # 初始任务下发入口
    def start_requests(self):
        for i in range(1, 2):
            # 请求
            yield feapder.Request("http://bang.dangdang.com/books/ebooks/98.01.00.00.00.00-24hours-0-0-1-{}/".format(i))

    def parse(self, request, response):
        article_list = response.xpath('//ul[@class="bang_list clearfix bang_list_mode"]/li')
        print(article_list)
        for article in article_list:
            list_num = article.xpath(
                './div[@class="list_num "]/text()|./div[@class="list_num red"]/text()').extract_first()
            list_num = list_num.replace(".", "").strip()
            details_url = article.xpath('./div[@class="pic"]/a[1]/@href').extract_first()
            img_src = article.xpath('./div[@class="pic"]/a[1]/img[1]/@src').extract_first()
            tuijian = article.xpath('./div[@class="star"]/span[@class="tuijian"]/text()').extract_first()
            tuijian = tuijian.replace("推荐", "") if len(tuijian) > 0 else ""
            publisher_author_nationality = article.xpath('./div[@class="publisher_info"][1]/text()').extract_first()
            publisher_author_nationality = publisher_author_nationality if publisher_author_nationality is not None and len(
                publisher_author_nationality) > 0 else ""
            publisher_author_name = article.xpath('./div[@class="publisher_info"][1]/a[1]/text()').extract_first()
            publisher = article.xpath('./div[@class="publisher_info"][2]/a[1]/text()').extract_first()
            price = article.xpath('./div[@class="price"]/p[1]/span[1]/text()').extract_first()

            book = DangDangBook(list_num, details_url, img_src, tuijian, publisher_author_nationality,
                                publisher_author_name, publisher, price, "", "")

            yield feapder.Request(
                details_url, callback=self.parse_detail, book=book,
            )  # callback 为回调函数

    def parse_detail(self, request, response):
        """
        解析详情
        """
        # 解析正文
        title_descript = response.xpath(
            'string(//p[@class="title_descript"][1])'
        ).extract_first()  # string 表达式是取某个标签下的文本，包括子标签文本
        descript = response.xpath(
            'string(//div[@id="newEditModule"][1])'
        ).extract_first()  # string 表达式是取某个标签下的文本，包括子标签文本
        book = request.book
        book.set_title_descript(title_descript)
        book.set_descript(descript)

        item = Item()  # 声明一个item
        item.table_name = "dangdangbook"  # 指定存储的表名
        item.num = book.num  # 给item属性赋值
        item.details_url = book.details_url
        item.img_url = book.img_url
        item.tuijian = book.tuijian
        item.publisher_author_nationality = book.publisher_author_nationality
        item.publisher_author_name = book.publisher_author_name
        item.publisher = book.publisher
        item.prce = book.prce
        item.title_descript = book.title_descript
        item.descript = book.descript
        yield item  # 返回item， item会自动批量入库


if __name__ == "__main__":
    Spider().start()
